{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lyp/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n",
      "/home/lyp/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/tensorflow/mnist/input_data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32, shape=[])\n",
    "keep_prob = tf.placeholder(tf.float32)#\n",
    "examNum = mnist.train.num_examples // 100  #除法运算// 返回商的整数部分，抛弃余数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_lay(lastoutput,row, col, actfun=None):\n",
    "    w = tf.Variable(tf.truncated_normal([row, col], stddev=0.1))\n",
    "    b = tf.Variable(tf.zeros([col]) + 0.1)\n",
    "    output = tf.matmul(lastoutput, w) + b\n",
    "    #tf.add_to_collection('losses', tf.contrib.layers.l2_regularizer(0.0024)(w))\n",
    "    if actfun != None:\n",
    "        output = actfun(output)\n",
    "        output = tf.nn.dropout(output, keep_prob)\n",
    "    return output, w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_def(num1, num2):\n",
    "    y1, w1 = add_lay(x, 784, num1, tf.nn.tanh)\n",
    "    y2, w2 = add_lay(y1, num1, num2, tf.nn.2)\n",
    "    y, w3  = add_lay(y2, num2, 10)\n",
    "    cross_entropy2 = tf.reduce_mean(\n",
    "        tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)\n",
    "    ) \n",
    "    #tf.add_to_collection('losses', cross_entropy2)\n",
    "    #loss = tf.add_n(tf.get_collection('losses'))\n",
    "    \n",
    "    \n",
    "    train_step = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cross_entropy2)\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        \n",
    "    #下面开始训练\n",
    "    sess = tf.Session()\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    sess.run(init_op)\n",
    "    # Train\n",
    "    maxstep =80\n",
    "    for step in range(maxstep):\n",
    "        \n",
    "        #lr = rate[math.floor(step / 4000)]\n",
    "        lr = 0.001 * (0.95 ** step)\n",
    "        \n",
    "        for i in range(examNum):\n",
    "            batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "            sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr, keep_prob:0.8})\n",
    "\n",
    "        preloss = sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob:1})\n",
    "        print(\"{0}:learning_rate:{1} test loss:{2}\".format(step, lr, preloss))\n",
    "        if preloss > 0.9815:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:learning_rate:0.001 test loss:0.9452999830245972\n",
      "1:learning_rate:0.00095 test loss:0.9581000208854675\n",
      "2:learning_rate:0.0009025 test loss:0.9668999910354614\n",
      "3:learning_rate:0.000857375 test loss:0.9682000279426575\n",
      "4:learning_rate:0.0008145062499999999 test loss:0.972599983215332\n",
      "5:learning_rate:0.0007737809374999998 test loss:0.9750999808311462\n",
      "6:learning_rate:0.0007350918906249999 test loss:0.9763000011444092\n",
      "7:learning_rate:0.0006983372960937497 test loss:0.9764000177383423\n",
      "8:learning_rate:0.0006634204312890623 test loss:0.9779000282287598\n",
      "9:learning_rate:0.0006302494097246091 test loss:0.9789999723434448\n",
      "10:learning_rate:0.0005987369392383787 test loss:0.978600025177002\n",
      "11:learning_rate:0.0005688000922764596 test loss:0.9793999791145325\n",
      "12:learning_rate:0.0005403600876626366 test loss:0.9794999957084656\n",
      "13:learning_rate:0.0005133420832795048 test loss:0.9796000123023987\n",
      "14:learning_rate:0.00048767497911552955 test loss:0.9801999926567078\n",
      "15:learning_rate:0.000463291230159753 test loss:0.9811999797821045\n",
      "16:learning_rate:0.00044012666865176535 test loss:0.9793999791145325\n",
      "17:learning_rate:0.0004181203352191771 test loss:0.9803000092506409\n",
      "18:learning_rate:0.0003972143184582182 test loss:0.9799000024795532\n",
      "19:learning_rate:0.00037735360253530727 test loss:0.9803000092506409\n",
      "20:learning_rate:0.0003584859224085419 test loss:0.9800000190734863\n",
      "21:learning_rate:0.0003405616262881148 test loss:0.980400025844574\n",
      "22:learning_rate:0.000323533544973709 test loss:0.9815000295639038\n"
     ]
    }
   ],
   "source": [
    "model_def(200, 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "通过这题的测试\n",
    "1.数据的初始化时， w使用 truncated_normal 这种方式 按标准差0.1来构建 比随见要好， bias不能使用全0，这样梯度下降时更新会出错\n",
    "2.防止过拟合使用dropout， 随机删除一些结点， 能防止过拟合，并且机算计算速度更快， 一般取0.5 或 0.8， 在预测时使用1\n",
    "3.求解w 使用AdamOptimizer 根据损失函数对每个参数的梯度的一阶矩估计和二阶矩估计动态调整针对于每个参数的学习速率\n",
    "4.最后学习率， 前期希望学习率够大 进步够快， 但是 逼近最值时 学习率小一点，逐步逼近最小值\n",
    "5.最后的训练 使用两个循环， 因为， 对每个数据 先使用 够大的学习率 初步学习一遍， 在使用小一点的学习率 细调，\n",
    "    这时候，自己指定学习率 就不如 动态调整学习率\n",
    "5.激活函数 使用tanh 比 sigmod要好\n",
    "\n",
    "雷点：\n",
    "1.正则使用 l2正则，l2正则的惩罚因子 影响很大， 最后预测结果在0.97左右徘徊，上不了0.98\n",
    "2.随即梯度下降， 可以发现，w变化会非常剧烈， 在极值附近徘徊，很难达到极值\n",
    "3.多层神经网络使用随即梯度下降 + l2，最后可以发现， 参数最好时 结果在0.978左右，最好的一次0.9796。。。就是上不了0.98， \n",
    "  可能是训练不够，反正 没有drop + Adam 效果好\n",
    "\n",
    "'''"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
